package com.ruby.bigtable.spark;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;

public class StringCount {
	public static void main(String[] args) {
		SparkConf conf = new SparkConf().setMaster("local").setAppName("string count");
		@SuppressWarnings("resource")
		JavaSparkContext sc = new JavaSparkContext(conf);
		
		JavaRDD<String> lines = sc.textFile("E:\\1111.txt", 5);
		JavaRDD<Integer> rowCountRdd = lines.map(new Function<String, Integer>() {

			/**
			 * 
			 */
			private static final long serialVersionUID = 1L;

			public Integer call(String line) throws Exception {
				// TODO Auto-generated method stub
				System.out.println(line.length());
				return line.length();
			}

		});
		
		int totalNum = rowCountRdd.reduce(new Function2<Integer, Integer, Integer>() {
			
			/**
			 * 
			 */
			private static final long serialVersionUID = 1L;

			public Integer call(Integer num1, Integer num2) throws Exception {
				// TODO Auto-generated method stub
				return num1+num2;
			}
		});
		
		System.out.println("单词总数："+totalNum);
		
		sc.stop();
	}
}
